{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "## 4.3.1 CBOW模型的实现"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2023-05-06T16:31:54.093339200Z",
     "start_time": "2023-05-06T16:31:54.071285Z"
    }
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "sys.path.append('..')\n",
    "import numpy as np\n",
    "from common.layers import Embedding\n",
    "from negative_sampling_layer import NegativeSamplingLoss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "这个初始化方法有 4 个参数。vocab_size 是词汇量，hidden_size 是中间层的神经元个数，corpus 是单词 ID 列表。另外，通过 window_size 指定上下文的大小，即上下文包含多少个周围单词。如果 window_size 是 2，则目标词的左右 2 个单词（共 4 个单词）将成为上下文\n",
    "\"\"\""
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "class CBOW:\n",
    "    def __init__(self, vocab_size, hidden_size, window_size, corpus):\n",
    "        V, H = vocab_size, hidden_size\n",
    "\n",
    "        # 初始化权重\n",
    "        W_in = 0.01 * np.random.randn(V, H).astype('f')\n",
    "        W_out = 0.01 * np.random.randn(V, H).astype('f')\n",
    "\n",
    "        # 生成层\n",
    "        self.in_layers = []\n",
    "        for i in range(2 * window_size):\n",
    "            layer = Embedding(W_in)  # 使用Embedding层\n",
    "            self.in_layers.append(layer)\n",
    "        self.ns_loss = NegativeSamplingLoss(W_out, corpus, power=0.75,\n",
    "                                            sample_size=5)\n",
    "\n",
    "        # 将所有的权重和梯度整理到列表中\n",
    "        layers = self.in_layers + [self.ns_loss]\n",
    "        self.params, self.grads = [], []\n",
    "        for layer in layers:\n",
    "            self.params += layer.params\n",
    "            self.grads += layer.grads\n",
    "\n",
    "        # 将单词的分布式表示设置为成员变量\n",
    "        self.word_vecs = W_in\n",
    "\n",
    "    def forward(self, contexts, target):\n",
    "        h = 0\n",
    "        for i, layer in enumerate(self.in_layers):\n",
    "            h += layer.forward(contexts[:, i])\n",
    "        h *= 1 / len(self.in_layers)\n",
    "        loss = self.ns_loss.forward(h, target)\n",
    "        return loss\n",
    "\n",
    "    def backward(self, dout=1):\n",
    "        dout = self.ns_loss.backward(dout)\n",
    "        dout *= 1 / len(self.in_layers)\n",
    "        for layer in self.in_layers:\n",
    "            layer.backward(dout)\n",
    "        return None"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-06T16:35:36.254230200Z",
     "start_time": "2023-05-06T16:35:36.238189600Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "sys.path.append('..')\n",
    "import numpy as np\n",
    "from common import config\n",
    "\n",
    "# 在用GPU运行时，请打开下面的注释（需要cupy）\n",
    "# ===============================================\n",
    "config.GPU = False\n",
    "# ===============================================\n",
    "import pickle\n",
    "from common.trainer import Trainer\n",
    "from common.optimizer import Adam\n",
    "from common.util import create_contexts_target, to_cpu, to_gpu\n",
    "from dataset import ptb"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-06T16:43:44.014657400Z",
     "start_time": "2023-05-06T16:43:43.998659400Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [],
   "source": [
    "window_size = 5\n",
    "hidden_size = 100\n",
    "batch_size = 100\n",
    "max_epoch = 10"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-06T16:43:44.372821400Z",
     "start_time": "2023-05-06T16:43:44.362823Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [],
   "source": [
    "# 读入数据\n",
    "corpus, word_to_id, id_to_word = ptb.load_data('train')\n",
    "vocab_size = len(word_to_id)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-06T16:43:44.749964Z",
     "start_time": "2023-05-06T16:43:44.726965700Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False\n"
     ]
    }
   ],
   "source": [
    "print(config.GPU)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-06T16:43:45.660680Z",
     "start_time": "2023-05-06T16:43:45.635661900Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "outputs": [],
   "source": [
    "contexts, target = create_contexts_target(corpus, window_size)\n",
    "if config.GPU:\n",
    "    contexts, target = to_gpu(contexts), to_gpu(target)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-06T16:43:49.360193300Z",
     "start_time": "2023-05-06T16:43:46.604194900Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "outputs": [],
   "source": [
    "# 生成模型等\n",
    "model = CBOW(vocab_size, hidden_size, window_size, corpus)\n",
    "optimizer = Adam()\n",
    "trainer = Trainer(model, optimizer)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-06T16:43:50.553811Z",
     "start_time": "2023-05-06T16:43:50.293242900Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch 1 |  iter 1 / 9295 | time 0[s] | loss 4.16\n",
      "| epoch 1 |  iter 21 / 9295 | time 1[s] | loss 4.16\n",
      "| epoch 1 |  iter 41 / 9295 | time 2[s] | loss 4.15\n",
      "| epoch 1 |  iter 61 / 9295 | time 3[s] | loss 4.13\n",
      "| epoch 1 |  iter 81 / 9295 | time 4[s] | loss 4.06\n",
      "| epoch 1 |  iter 101 / 9295 | time 5[s] | loss 3.94\n",
      "| epoch 1 |  iter 121 / 9295 | time 6[s] | loss 3.79\n",
      "| epoch 1 |  iter 141 / 9295 | time 7[s] | loss 3.65\n",
      "| epoch 1 |  iter 161 / 9295 | time 8[s] | loss 3.50\n",
      "| epoch 1 |  iter 181 / 9295 | time 9[s] | loss 3.38\n",
      "| epoch 1 |  iter 201 / 9295 | time 10[s] | loss 3.26\n",
      "| epoch 1 |  iter 221 / 9295 | time 11[s] | loss 3.15\n",
      "| epoch 1 |  iter 241 / 9295 | time 12[s] | loss 3.10\n",
      "| epoch 1 |  iter 261 / 9295 | time 13[s] | loss 3.02\n",
      "| epoch 1 |  iter 281 / 9295 | time 14[s] | loss 2.97\n",
      "| epoch 1 |  iter 301 / 9295 | time 15[s] | loss 2.92\n",
      "| epoch 1 |  iter 321 / 9295 | time 16[s] | loss 2.89\n",
      "| epoch 1 |  iter 341 / 9295 | time 17[s] | loss 2.85\n",
      "| epoch 1 |  iter 361 / 9295 | time 19[s] | loss 2.81\n",
      "| epoch 1 |  iter 381 / 9295 | time 20[s] | loss 2.79\n",
      "| epoch 1 |  iter 401 / 9295 | time 21[s] | loss 2.76\n",
      "| epoch 1 |  iter 421 / 9295 | time 22[s] | loss 2.75\n",
      "| epoch 1 |  iter 441 / 9295 | time 23[s] | loss 2.73\n",
      "| epoch 1 |  iter 461 / 9295 | time 24[s] | loss 2.71\n",
      "| epoch 1 |  iter 481 / 9295 | time 25[s] | loss 2.68\n",
      "| epoch 1 |  iter 501 / 9295 | time 26[s] | loss 2.69\n",
      "| epoch 1 |  iter 521 / 9295 | time 27[s] | loss 2.68\n",
      "| epoch 1 |  iter 541 / 9295 | time 28[s] | loss 2.68\n",
      "| epoch 1 |  iter 561 / 9295 | time 29[s] | loss 2.65\n",
      "| epoch 1 |  iter 581 / 9295 | time 30[s] | loss 2.67\n",
      "| epoch 1 |  iter 601 / 9295 | time 31[s] | loss 2.63\n",
      "| epoch 1 |  iter 621 / 9295 | time 32[s] | loss 2.62\n",
      "| epoch 1 |  iter 641 / 9295 | time 33[s] | loss 2.60\n",
      "| epoch 1 |  iter 661 / 9295 | time 34[s] | loss 2.61\n",
      "| epoch 1 |  iter 681 / 9295 | time 35[s] | loss 2.62\n",
      "| epoch 1 |  iter 701 / 9295 | time 36[s] | loss 2.59\n",
      "| epoch 1 |  iter 721 / 9295 | time 37[s] | loss 2.60\n",
      "| epoch 1 |  iter 741 / 9295 | time 38[s] | loss 2.60\n",
      "| epoch 1 |  iter 761 / 9295 | time 39[s] | loss 2.59\n",
      "| epoch 1 |  iter 781 / 9295 | time 40[s] | loss 2.57\n",
      "| epoch 1 |  iter 801 / 9295 | time 42[s] | loss 2.57\n",
      "| epoch 1 |  iter 821 / 9295 | time 42[s] | loss 2.61\n",
      "| epoch 1 |  iter 841 / 9295 | time 43[s] | loss 2.59\n",
      "| epoch 1 |  iter 861 / 9295 | time 44[s] | loss 2.56\n",
      "| epoch 1 |  iter 881 / 9295 | time 45[s] | loss 2.59\n",
      "| epoch 1 |  iter 901 / 9295 | time 46[s] | loss 2.60\n",
      "| epoch 1 |  iter 921 / 9295 | time 47[s] | loss 2.60\n",
      "| epoch 1 |  iter 941 / 9295 | time 48[s] | loss 2.56\n",
      "| epoch 1 |  iter 961 / 9295 | time 49[s] | loss 2.56\n",
      "| epoch 1 |  iter 981 / 9295 | time 50[s] | loss 2.56\n",
      "| epoch 1 |  iter 1001 / 9295 | time 51[s] | loss 2.55\n",
      "| epoch 1 |  iter 1021 / 9295 | time 52[s] | loss 2.54\n",
      "| epoch 1 |  iter 1041 / 9295 | time 53[s] | loss 2.53\n",
      "| epoch 1 |  iter 1061 / 9295 | time 54[s] | loss 2.58\n",
      "| epoch 1 |  iter 1081 / 9295 | time 55[s] | loss 2.55\n",
      "| epoch 1 |  iter 1101 / 9295 | time 56[s] | loss 2.54\n",
      "| epoch 1 |  iter 1121 / 9295 | time 57[s] | loss 2.52\n",
      "| epoch 1 |  iter 1141 / 9295 | time 57[s] | loss 2.54\n",
      "| epoch 1 |  iter 1161 / 9295 | time 58[s] | loss 2.53\n",
      "| epoch 1 |  iter 1181 / 9295 | time 59[s] | loss 2.56\n",
      "| epoch 1 |  iter 1201 / 9295 | time 60[s] | loss 2.53\n",
      "| epoch 1 |  iter 1221 / 9295 | time 61[s] | loss 2.55\n",
      "| epoch 1 |  iter 1241 / 9295 | time 62[s] | loss 2.50\n",
      "| epoch 1 |  iter 1261 / 9295 | time 63[s] | loss 2.52\n",
      "| epoch 1 |  iter 1281 / 9295 | time 64[s] | loss 2.52\n",
      "| epoch 1 |  iter 1301 / 9295 | time 65[s] | loss 2.54\n",
      "| epoch 1 |  iter 1321 / 9295 | time 66[s] | loss 2.54\n",
      "| epoch 1 |  iter 1341 / 9295 | time 67[s] | loss 2.53\n",
      "| epoch 1 |  iter 1361 / 9295 | time 68[s] | loss 2.53\n",
      "| epoch 1 |  iter 1381 / 9295 | time 69[s] | loss 2.53\n",
      "| epoch 1 |  iter 1401 / 9295 | time 70[s] | loss 2.51\n",
      "| epoch 1 |  iter 1421 / 9295 | time 71[s] | loss 2.51\n",
      "| epoch 1 |  iter 1441 / 9295 | time 72[s] | loss 2.49\n",
      "| epoch 1 |  iter 1461 / 9295 | time 73[s] | loss 2.49\n",
      "| epoch 1 |  iter 1481 / 9295 | time 74[s] | loss 2.50\n",
      "| epoch 1 |  iter 1501 / 9295 | time 75[s] | loss 2.49\n",
      "| epoch 1 |  iter 1521 / 9295 | time 76[s] | loss 2.51\n",
      "| epoch 1 |  iter 1541 / 9295 | time 76[s] | loss 2.50\n",
      "| epoch 1 |  iter 1561 / 9295 | time 77[s] | loss 2.49\n",
      "| epoch 1 |  iter 1581 / 9295 | time 78[s] | loss 2.51\n",
      "| epoch 1 |  iter 1601 / 9295 | time 79[s] | loss 2.51\n",
      "| epoch 1 |  iter 1621 / 9295 | time 80[s] | loss 2.53\n",
      "| epoch 1 |  iter 1641 / 9295 | time 81[s] | loss 2.54\n",
      "| epoch 1 |  iter 1661 / 9295 | time 82[s] | loss 2.49\n",
      "| epoch 1 |  iter 1681 / 9295 | time 83[s] | loss 2.53\n",
      "| epoch 1 |  iter 1701 / 9295 | time 84[s] | loss 2.49\n",
      "| epoch 1 |  iter 1721 / 9295 | time 85[s] | loss 2.54\n",
      "| epoch 1 |  iter 1741 / 9295 | time 86[s] | loss 2.49\n",
      "| epoch 1 |  iter 1761 / 9295 | time 87[s] | loss 2.52\n",
      "| epoch 1 |  iter 1781 / 9295 | time 88[s] | loss 2.49\n",
      "| epoch 1 |  iter 1801 / 9295 | time 89[s] | loss 2.51\n",
      "| epoch 1 |  iter 1821 / 9295 | time 90[s] | loss 2.48\n",
      "| epoch 1 |  iter 1841 / 9295 | time 91[s] | loss 2.50\n",
      "| epoch 1 |  iter 1861 / 9295 | time 92[s] | loss 2.45\n",
      "| epoch 1 |  iter 1881 / 9295 | time 93[s] | loss 2.46\n",
      "| epoch 1 |  iter 1901 / 9295 | time 94[s] | loss 2.51\n",
      "| epoch 1 |  iter 1921 / 9295 | time 95[s] | loss 2.52\n",
      "| epoch 1 |  iter 1941 / 9295 | time 96[s] | loss 2.49\n",
      "| epoch 1 |  iter 1961 / 9295 | time 97[s] | loss 2.50\n",
      "| epoch 1 |  iter 1981 / 9295 | time 98[s] | loss 2.49\n",
      "| epoch 1 |  iter 2001 / 9295 | time 99[s] | loss 2.50\n",
      "| epoch 1 |  iter 2021 / 9295 | time 100[s] | loss 2.51\n",
      "| epoch 1 |  iter 2041 / 9295 | time 101[s] | loss 2.48\n",
      "| epoch 1 |  iter 2061 / 9295 | time 102[s] | loss 2.48\n",
      "| epoch 1 |  iter 2081 / 9295 | time 102[s] | loss 2.49\n",
      "| epoch 1 |  iter 2101 / 9295 | time 103[s] | loss 2.50\n",
      "| epoch 1 |  iter 2121 / 9295 | time 104[s] | loss 2.48\n",
      "| epoch 1 |  iter 2141 / 9295 | time 105[s] | loss 2.48\n",
      "| epoch 1 |  iter 2161 / 9295 | time 106[s] | loss 2.46\n",
      "| epoch 1 |  iter 2181 / 9295 | time 107[s] | loss 2.48\n",
      "| epoch 1 |  iter 2201 / 9295 | time 108[s] | loss 2.48\n",
      "| epoch 1 |  iter 2221 / 9295 | time 109[s] | loss 2.46\n",
      "| epoch 1 |  iter 2241 / 9295 | time 110[s] | loss 2.47\n",
      "| epoch 1 |  iter 2261 / 9295 | time 111[s] | loss 2.48\n",
      "| epoch 1 |  iter 2281 / 9295 | time 112[s] | loss 2.48\n",
      "| epoch 1 |  iter 2301 / 9295 | time 113[s] | loss 2.48\n",
      "| epoch 1 |  iter 2321 / 9295 | time 114[s] | loss 2.49\n",
      "| epoch 1 |  iter 2341 / 9295 | time 115[s] | loss 2.49\n",
      "| epoch 1 |  iter 2361 / 9295 | time 116[s] | loss 2.49\n",
      "| epoch 1 |  iter 2381 / 9295 | time 116[s] | loss 2.49\n",
      "| epoch 1 |  iter 2401 / 9295 | time 117[s] | loss 2.46\n",
      "| epoch 1 |  iter 2421 / 9295 | time 118[s] | loss 2.47\n",
      "| epoch 1 |  iter 2441 / 9295 | time 119[s] | loss 2.47\n",
      "| epoch 1 |  iter 2461 / 9295 | time 120[s] | loss 2.48\n",
      "| epoch 1 |  iter 2481 / 9295 | time 122[s] | loss 2.49\n",
      "| epoch 1 |  iter 2501 / 9295 | time 122[s] | loss 2.48\n",
      "| epoch 1 |  iter 2521 / 9295 | time 123[s] | loss 2.47\n",
      "| epoch 1 |  iter 2541 / 9295 | time 124[s] | loss 2.48\n",
      "| epoch 1 |  iter 2561 / 9295 | time 125[s] | loss 2.48\n",
      "| epoch 1 |  iter 2581 / 9295 | time 126[s] | loss 2.45\n",
      "| epoch 1 |  iter 2601 / 9295 | time 127[s] | loss 2.46\n",
      "| epoch 1 |  iter 2621 / 9295 | time 128[s] | loss 2.48\n",
      "| epoch 1 |  iter 2641 / 9295 | time 129[s] | loss 2.48\n",
      "| epoch 1 |  iter 2661 / 9295 | time 130[s] | loss 2.48\n",
      "| epoch 1 |  iter 2681 / 9295 | time 131[s] | loss 2.44\n",
      "| epoch 1 |  iter 2701 / 9295 | time 132[s] | loss 2.46\n",
      "| epoch 1 |  iter 2721 / 9295 | time 133[s] | loss 2.45\n",
      "| epoch 1 |  iter 2741 / 9295 | time 134[s] | loss 2.43\n",
      "| epoch 1 |  iter 2761 / 9295 | time 135[s] | loss 2.45\n",
      "| epoch 1 |  iter 2781 / 9295 | time 136[s] | loss 2.47\n",
      "| epoch 1 |  iter 2801 / 9295 | time 137[s] | loss 2.47\n",
      "| epoch 1 |  iter 2821 / 9295 | time 138[s] | loss 2.45\n",
      "| epoch 1 |  iter 2841 / 9295 | time 139[s] | loss 2.42\n",
      "| epoch 1 |  iter 2861 / 9295 | time 140[s] | loss 2.48\n",
      "| epoch 1 |  iter 2881 / 9295 | time 141[s] | loss 2.46\n",
      "| epoch 1 |  iter 2901 / 9295 | time 142[s] | loss 2.44\n",
      "| epoch 1 |  iter 2921 / 9295 | time 143[s] | loss 2.45\n",
      "| epoch 1 |  iter 2941 / 9295 | time 144[s] | loss 2.45\n",
      "| epoch 1 |  iter 2961 / 9295 | time 145[s] | loss 2.44\n",
      "| epoch 1 |  iter 2981 / 9295 | time 146[s] | loss 2.48\n",
      "| epoch 1 |  iter 3001 / 9295 | time 147[s] | loss 2.44\n",
      "| epoch 1 |  iter 3021 / 9295 | time 148[s] | loss 2.46\n",
      "| epoch 1 |  iter 3041 / 9295 | time 149[s] | loss 2.44\n",
      "| epoch 1 |  iter 3061 / 9295 | time 150[s] | loss 2.43\n",
      "| epoch 1 |  iter 3081 / 9295 | time 151[s] | loss 2.44\n",
      "| epoch 1 |  iter 3101 / 9295 | time 152[s] | loss 2.45\n",
      "| epoch 1 |  iter 3121 / 9295 | time 153[s] | loss 2.42\n",
      "| epoch 1 |  iter 3141 / 9295 | time 155[s] | loss 2.40\n",
      "| epoch 1 |  iter 3161 / 9295 | time 156[s] | loss 2.44\n",
      "| epoch 1 |  iter 3181 / 9295 | time 157[s] | loss 2.47\n",
      "| epoch 1 |  iter 3201 / 9295 | time 158[s] | loss 2.46\n",
      "| epoch 1 |  iter 3221 / 9295 | time 159[s] | loss 2.43\n",
      "| epoch 1 |  iter 3241 / 9295 | time 160[s] | loss 2.44\n",
      "| epoch 1 |  iter 3261 / 9295 | time 161[s] | loss 2.48\n",
      "| epoch 1 |  iter 3281 / 9295 | time 162[s] | loss 2.42\n",
      "| epoch 1 |  iter 3301 / 9295 | time 163[s] | loss 2.44\n",
      "| epoch 1 |  iter 3321 / 9295 | time 164[s] | loss 2.45\n",
      "| epoch 1 |  iter 3341 / 9295 | time 165[s] | loss 2.44\n",
      "| epoch 1 |  iter 3361 / 9295 | time 166[s] | loss 2.40\n",
      "| epoch 1 |  iter 3381 / 9295 | time 167[s] | loss 2.43\n",
      "| epoch 1 |  iter 3401 / 9295 | time 168[s] | loss 2.43\n",
      "| epoch 1 |  iter 3421 / 9295 | time 169[s] | loss 2.40\n",
      "| epoch 1 |  iter 3441 / 9295 | time 170[s] | loss 2.44\n",
      "| epoch 1 |  iter 3461 / 9295 | time 171[s] | loss 2.43\n",
      "| epoch 1 |  iter 3481 / 9295 | time 172[s] | loss 2.44\n",
      "| epoch 1 |  iter 3501 / 9295 | time 173[s] | loss 2.43\n",
      "| epoch 1 |  iter 3521 / 9295 | time 174[s] | loss 2.43\n",
      "| epoch 1 |  iter 3541 / 9295 | time 175[s] | loss 2.40\n",
      "| epoch 1 |  iter 3561 / 9295 | time 176[s] | loss 2.43\n",
      "| epoch 1 |  iter 3581 / 9295 | time 177[s] | loss 2.42\n",
      "| epoch 1 |  iter 3601 / 9295 | time 178[s] | loss 2.43\n",
      "| epoch 1 |  iter 3621 / 9295 | time 179[s] | loss 2.41\n",
      "| epoch 1 |  iter 3641 / 9295 | time 180[s] | loss 2.42\n",
      "| epoch 1 |  iter 3661 / 9295 | time 181[s] | loss 2.42\n",
      "| epoch 1 |  iter 3681 / 9295 | time 182[s] | loss 2.40\n",
      "| epoch 1 |  iter 3701 / 9295 | time 183[s] | loss 2.39\n",
      "| epoch 1 |  iter 3721 / 9295 | time 184[s] | loss 2.43\n",
      "| epoch 1 |  iter 3741 / 9295 | time 185[s] | loss 2.39\n",
      "| epoch 1 |  iter 3761 / 9295 | time 186[s] | loss 2.39\n",
      "| epoch 1 |  iter 3781 / 9295 | time 187[s] | loss 2.43\n",
      "| epoch 1 |  iter 3801 / 9295 | time 188[s] | loss 2.41\n",
      "| epoch 1 |  iter 3821 / 9295 | time 189[s] | loss 2.40\n",
      "| epoch 1 |  iter 3841 / 9295 | time 190[s] | loss 2.39\n",
      "| epoch 1 |  iter 3861 / 9295 | time 191[s] | loss 2.41\n",
      "| epoch 1 |  iter 3881 / 9295 | time 192[s] | loss 2.44\n",
      "| epoch 1 |  iter 3901 / 9295 | time 193[s] | loss 2.43\n",
      "| epoch 1 |  iter 3921 / 9295 | time 194[s] | loss 2.39\n",
      "| epoch 1 |  iter 3941 / 9295 | time 195[s] | loss 2.39\n",
      "| epoch 1 |  iter 3961 / 9295 | time 197[s] | loss 2.39\n",
      "| epoch 1 |  iter 3981 / 9295 | time 198[s] | loss 2.37\n",
      "| epoch 1 |  iter 4001 / 9295 | time 199[s] | loss 2.41\n",
      "| epoch 1 |  iter 4021 / 9295 | time 200[s] | loss 2.40\n",
      "| epoch 1 |  iter 4041 / 9295 | time 201[s] | loss 2.36\n",
      "| epoch 1 |  iter 4061 / 9295 | time 202[s] | loss 2.42\n",
      "| epoch 1 |  iter 4081 / 9295 | time 203[s] | loss 2.39\n",
      "| epoch 1 |  iter 4101 / 9295 | time 204[s] | loss 2.40\n",
      "| epoch 1 |  iter 4121 / 9295 | time 205[s] | loss 2.38\n",
      "| epoch 1 |  iter 4141 / 9295 | time 206[s] | loss 2.39\n",
      "| epoch 1 |  iter 4161 / 9295 | time 207[s] | loss 2.37\n",
      "| epoch 1 |  iter 4181 / 9295 | time 208[s] | loss 2.36\n",
      "| epoch 1 |  iter 4201 / 9295 | time 209[s] | loss 2.39\n",
      "| epoch 1 |  iter 4221 / 9295 | time 210[s] | loss 2.38\n",
      "| epoch 1 |  iter 4241 / 9295 | time 211[s] | loss 2.37\n",
      "| epoch 1 |  iter 4261 / 9295 | time 212[s] | loss 2.37\n",
      "| epoch 1 |  iter 4281 / 9295 | time 213[s] | loss 2.43\n",
      "| epoch 1 |  iter 4301 / 9295 | time 214[s] | loss 2.40\n",
      "| epoch 1 |  iter 4321 / 9295 | time 215[s] | loss 2.40\n",
      "| epoch 1 |  iter 4341 / 9295 | time 216[s] | loss 2.39\n",
      "| epoch 1 |  iter 4361 / 9295 | time 217[s] | loss 2.38\n",
      "| epoch 1 |  iter 4381 / 9295 | time 218[s] | loss 2.39\n",
      "| epoch 1 |  iter 4401 / 9295 | time 219[s] | loss 2.36\n",
      "| epoch 1 |  iter 4421 / 9295 | time 220[s] | loss 2.37\n",
      "| epoch 1 |  iter 4441 / 9295 | time 221[s] | loss 2.36\n",
      "| epoch 1 |  iter 4461 / 9295 | time 222[s] | loss 2.38\n",
      "| epoch 1 |  iter 4481 / 9295 | time 223[s] | loss 2.38\n",
      "| epoch 1 |  iter 4501 / 9295 | time 225[s] | loss 2.37\n",
      "| epoch 1 |  iter 4521 / 9295 | time 226[s] | loss 2.39\n",
      "| epoch 1 |  iter 4541 / 9295 | time 227[s] | loss 2.38\n",
      "| epoch 1 |  iter 4561 / 9295 | time 228[s] | loss 2.38\n",
      "| epoch 1 |  iter 4581 / 9295 | time 229[s] | loss 2.35\n",
      "| epoch 1 |  iter 4601 / 9295 | time 230[s] | loss 2.42\n",
      "| epoch 1 |  iter 4621 / 9295 | time 231[s] | loss 2.36\n",
      "| epoch 1 |  iter 4641 / 9295 | time 232[s] | loss 2.40\n",
      "| epoch 1 |  iter 4661 / 9295 | time 233[s] | loss 2.37\n",
      "| epoch 1 |  iter 4681 / 9295 | time 234[s] | loss 2.39\n",
      "| epoch 1 |  iter 4701 / 9295 | time 235[s] | loss 2.35\n",
      "| epoch 1 |  iter 4721 / 9295 | time 236[s] | loss 2.35\n",
      "| epoch 1 |  iter 4741 / 9295 | time 236[s] | loss 2.38\n",
      "| epoch 1 |  iter 4761 / 9295 | time 237[s] | loss 2.37\n",
      "| epoch 1 |  iter 4781 / 9295 | time 238[s] | loss 2.39\n",
      "| epoch 1 |  iter 4801 / 9295 | time 239[s] | loss 2.38\n",
      "| epoch 1 |  iter 4821 / 9295 | time 240[s] | loss 2.37\n",
      "| epoch 1 |  iter 4841 / 9295 | time 241[s] | loss 2.38\n",
      "| epoch 1 |  iter 4861 / 9295 | time 242[s] | loss 2.39\n",
      "| epoch 1 |  iter 4881 / 9295 | time 243[s] | loss 2.36\n",
      "| epoch 1 |  iter 4901 / 9295 | time 244[s] | loss 2.38\n",
      "| epoch 1 |  iter 4921 / 9295 | time 245[s] | loss 2.35\n",
      "| epoch 1 |  iter 4941 / 9295 | time 246[s] | loss 2.36\n",
      "| epoch 1 |  iter 4961 / 9295 | time 247[s] | loss 2.35\n",
      "| epoch 1 |  iter 4981 / 9295 | time 248[s] | loss 2.35\n",
      "| epoch 1 |  iter 5001 / 9295 | time 249[s] | loss 2.34\n",
      "| epoch 1 |  iter 5021 / 9295 | time 250[s] | loss 2.32\n",
      "| epoch 1 |  iter 5041 / 9295 | time 251[s] | loss 2.39\n",
      "| epoch 1 |  iter 5061 / 9295 | time 252[s] | loss 2.34\n",
      "| epoch 1 |  iter 5081 / 9295 | time 253[s] | loss 2.34\n",
      "| epoch 1 |  iter 5101 / 9295 | time 253[s] | loss 2.32\n",
      "| epoch 1 |  iter 5121 / 9295 | time 254[s] | loss 2.36\n",
      "| epoch 1 |  iter 5141 / 9295 | time 255[s] | loss 2.37\n",
      "| epoch 1 |  iter 5161 / 9295 | time 256[s] | loss 2.34\n",
      "| epoch 1 |  iter 5181 / 9295 | time 257[s] | loss 2.33\n",
      "| epoch 1 |  iter 5201 / 9295 | time 258[s] | loss 2.34\n",
      "| epoch 1 |  iter 5221 / 9295 | time 259[s] | loss 2.35\n",
      "| epoch 1 |  iter 5241 / 9295 | time 260[s] | loss 2.36\n",
      "| epoch 1 |  iter 5261 / 9295 | time 261[s] | loss 2.36\n",
      "| epoch 1 |  iter 5281 / 9295 | time 262[s] | loss 2.37\n",
      "| epoch 1 |  iter 5301 / 9295 | time 263[s] | loss 2.32\n",
      "| epoch 1 |  iter 5321 / 9295 | time 264[s] | loss 2.33\n",
      "| epoch 1 |  iter 5341 / 9295 | time 265[s] | loss 2.39\n",
      "| epoch 1 |  iter 5361 / 9295 | time 266[s] | loss 2.34\n",
      "| epoch 1 |  iter 5381 / 9295 | time 267[s] | loss 2.36\n",
      "| epoch 1 |  iter 5401 / 9295 | time 268[s] | loss 2.37\n",
      "| epoch 1 |  iter 5421 / 9295 | time 269[s] | loss 2.35\n",
      "| epoch 1 |  iter 5441 / 9295 | time 270[s] | loss 2.35\n",
      "| epoch 1 |  iter 5461 / 9295 | time 271[s] | loss 2.32\n",
      "| epoch 1 |  iter 5481 / 9295 | time 271[s] | loss 2.35\n",
      "| epoch 1 |  iter 5501 / 9295 | time 272[s] | loss 2.35\n",
      "| epoch 1 |  iter 5521 / 9295 | time 273[s] | loss 2.33\n",
      "| epoch 1 |  iter 5541 / 9295 | time 274[s] | loss 2.33\n",
      "| epoch 1 |  iter 5561 / 9295 | time 275[s] | loss 2.30\n",
      "| epoch 1 |  iter 5581 / 9295 | time 276[s] | loss 2.38\n",
      "| epoch 1 |  iter 5601 / 9295 | time 277[s] | loss 2.33\n",
      "| epoch 1 |  iter 5621 / 9295 | time 278[s] | loss 2.36\n",
      "| epoch 1 |  iter 5641 / 9295 | time 279[s] | loss 2.36\n",
      "| epoch 1 |  iter 5661 / 9295 | time 280[s] | loss 2.34\n",
      "| epoch 1 |  iter 5681 / 9295 | time 281[s] | loss 2.32\n",
      "| epoch 1 |  iter 5701 / 9295 | time 282[s] | loss 2.36\n",
      "| epoch 1 |  iter 5721 / 9295 | time 283[s] | loss 2.33\n",
      "| epoch 1 |  iter 5741 / 9295 | time 284[s] | loss 2.31\n",
      "| epoch 1 |  iter 5761 / 9295 | time 285[s] | loss 2.33\n",
      "| epoch 1 |  iter 5781 / 9295 | time 286[s] | loss 2.32\n",
      "| epoch 1 |  iter 5801 / 9295 | time 286[s] | loss 2.33\n",
      "| epoch 1 |  iter 5821 / 9295 | time 287[s] | loss 2.33\n",
      "| epoch 1 |  iter 5841 / 9295 | time 288[s] | loss 2.34\n",
      "| epoch 1 |  iter 5861 / 9295 | time 289[s] | loss 2.32\n",
      "| epoch 1 |  iter 5881 / 9295 | time 290[s] | loss 2.32\n",
      "| epoch 1 |  iter 5901 / 9295 | time 291[s] | loss 2.34\n",
      "| epoch 1 |  iter 5921 / 9295 | time 292[s] | loss 2.36\n",
      "| epoch 1 |  iter 5941 / 9295 | time 293[s] | loss 2.33\n",
      "| epoch 1 |  iter 5961 / 9295 | time 294[s] | loss 2.30\n",
      "| epoch 1 |  iter 5981 / 9295 | time 295[s] | loss 2.33\n",
      "| epoch 1 |  iter 6001 / 9295 | time 296[s] | loss 2.33\n",
      "| epoch 1 |  iter 6021 / 9295 | time 297[s] | loss 2.30\n",
      "| epoch 1 |  iter 6041 / 9295 | time 298[s] | loss 2.32\n",
      "| epoch 1 |  iter 6061 / 9295 | time 299[s] | loss 2.34\n",
      "| epoch 1 |  iter 6081 / 9295 | time 300[s] | loss 2.32\n",
      "| epoch 1 |  iter 6101 / 9295 | time 301[s] | loss 2.32\n",
      "| epoch 1 |  iter 6121 / 9295 | time 302[s] | loss 2.32\n",
      "| epoch 1 |  iter 6141 / 9295 | time 303[s] | loss 2.35\n",
      "| epoch 1 |  iter 6161 / 9295 | time 304[s] | loss 2.31\n",
      "| epoch 1 |  iter 6181 / 9295 | time 305[s] | loss 2.34\n",
      "| epoch 1 |  iter 6201 / 9295 | time 306[s] | loss 2.30\n",
      "| epoch 1 |  iter 6221 / 9295 | time 307[s] | loss 2.31\n",
      "| epoch 1 |  iter 6241 / 9295 | time 308[s] | loss 2.32\n",
      "| epoch 1 |  iter 6261 / 9295 | time 309[s] | loss 2.29\n",
      "| epoch 1 |  iter 6281 / 9295 | time 310[s] | loss 2.31\n",
      "| epoch 1 |  iter 6301 / 9295 | time 311[s] | loss 2.32\n",
      "| epoch 1 |  iter 6321 / 9295 | time 312[s] | loss 2.31\n",
      "| epoch 1 |  iter 6341 / 9295 | time 313[s] | loss 2.29\n",
      "| epoch 1 |  iter 6361 / 9295 | time 314[s] | loss 2.30\n",
      "| epoch 1 |  iter 6381 / 9295 | time 315[s] | loss 2.33\n",
      "| epoch 1 |  iter 6401 / 9295 | time 316[s] | loss 2.34\n",
      "| epoch 1 |  iter 6421 / 9295 | time 317[s] | loss 2.28\n",
      "| epoch 1 |  iter 6441 / 9295 | time 318[s] | loss 2.30\n",
      "| epoch 1 |  iter 6461 / 9295 | time 319[s] | loss 2.31\n",
      "| epoch 1 |  iter 6481 / 9295 | time 320[s] | loss 2.32\n",
      "| epoch 1 |  iter 6501 / 9295 | time 320[s] | loss 2.31\n",
      "| epoch 1 |  iter 6521 / 9295 | time 321[s] | loss 2.31\n",
      "| epoch 1 |  iter 6541 / 9295 | time 322[s] | loss 2.27\n",
      "| epoch 1 |  iter 6561 / 9295 | time 323[s] | loss 2.28\n",
      "| epoch 1 |  iter 6581 / 9295 | time 324[s] | loss 2.27\n",
      "| epoch 1 |  iter 6601 / 9295 | time 325[s] | loss 2.29\n",
      "| epoch 1 |  iter 6621 / 9295 | time 326[s] | loss 2.28\n",
      "| epoch 1 |  iter 6641 / 9295 | time 327[s] | loss 2.29\n",
      "| epoch 1 |  iter 6661 / 9295 | time 328[s] | loss 2.33\n",
      "| epoch 1 |  iter 6681 / 9295 | time 329[s] | loss 2.30\n",
      "| epoch 1 |  iter 6701 / 9295 | time 330[s] | loss 2.31\n",
      "| epoch 1 |  iter 6721 / 9295 | time 331[s] | loss 2.28\n",
      "| epoch 1 |  iter 6741 / 9295 | time 332[s] | loss 2.33\n",
      "| epoch 1 |  iter 6761 / 9295 | time 333[s] | loss 2.31\n",
      "| epoch 1 |  iter 6781 / 9295 | time 334[s] | loss 2.29\n",
      "| epoch 1 |  iter 6801 / 9295 | time 335[s] | loss 2.29\n",
      "| epoch 1 |  iter 6821 / 9295 | time 336[s] | loss 2.25\n",
      "| epoch 1 |  iter 6841 / 9295 | time 337[s] | loss 2.32\n",
      "| epoch 1 |  iter 6861 / 9295 | time 338[s] | loss 2.32\n",
      "| epoch 1 |  iter 6881 / 9295 | time 339[s] | loss 2.28\n",
      "| epoch 1 |  iter 6901 / 9295 | time 340[s] | loss 2.31\n",
      "| epoch 1 |  iter 6921 / 9295 | time 341[s] | loss 2.27\n",
      "| epoch 1 |  iter 6941 / 9295 | time 342[s] | loss 2.29\n",
      "| epoch 1 |  iter 6961 / 9295 | time 343[s] | loss 2.26\n",
      "| epoch 1 |  iter 6981 / 9295 | time 344[s] | loss 2.30\n",
      "| epoch 1 |  iter 7001 / 9295 | time 345[s] | loss 2.26\n",
      "| epoch 1 |  iter 7021 / 9295 | time 346[s] | loss 2.31\n",
      "| epoch 1 |  iter 7041 / 9295 | time 347[s] | loss 2.28\n",
      "| epoch 1 |  iter 7061 / 9295 | time 348[s] | loss 2.30\n",
      "| epoch 1 |  iter 7081 / 9295 | time 349[s] | loss 2.26\n",
      "| epoch 1 |  iter 7101 / 9295 | time 350[s] | loss 2.27\n",
      "| epoch 1 |  iter 7121 / 9295 | time 351[s] | loss 2.28\n",
      "| epoch 1 |  iter 7141 / 9295 | time 352[s] | loss 2.28\n",
      "| epoch 1 |  iter 7161 / 9295 | time 353[s] | loss 2.30\n",
      "| epoch 1 |  iter 7181 / 9295 | time 354[s] | loss 2.24\n",
      "| epoch 1 |  iter 7201 / 9295 | time 355[s] | loss 2.28\n",
      "| epoch 1 |  iter 7221 / 9295 | time 356[s] | loss 2.30\n",
      "| epoch 1 |  iter 7241 / 9295 | time 358[s] | loss 2.31\n",
      "| epoch 1 |  iter 7261 / 9295 | time 359[s] | loss 2.27\n",
      "| epoch 1 |  iter 7281 / 9295 | time 360[s] | loss 2.26\n",
      "| epoch 1 |  iter 7301 / 9295 | time 361[s] | loss 2.27\n",
      "| epoch 1 |  iter 7321 / 9295 | time 362[s] | loss 2.28\n",
      "| epoch 1 |  iter 7341 / 9295 | time 363[s] | loss 2.24\n",
      "| epoch 1 |  iter 7361 / 9295 | time 364[s] | loss 2.28\n",
      "| epoch 1 |  iter 7381 / 9295 | time 365[s] | loss 2.29\n",
      "| epoch 1 |  iter 7401 / 9295 | time 366[s] | loss 2.27\n",
      "| epoch 1 |  iter 7421 / 9295 | time 367[s] | loss 2.24\n",
      "| epoch 1 |  iter 7441 / 9295 | time 368[s] | loss 2.29\n",
      "| epoch 1 |  iter 7461 / 9295 | time 369[s] | loss 2.25\n",
      "| epoch 1 |  iter 7481 / 9295 | time 370[s] | loss 2.26\n",
      "| epoch 1 |  iter 7501 / 9295 | time 371[s] | loss 2.31\n",
      "| epoch 1 |  iter 7521 / 9295 | time 372[s] | loss 2.26\n",
      "| epoch 1 |  iter 7541 / 9295 | time 373[s] | loss 2.27\n",
      "| epoch 1 |  iter 7561 / 9295 | time 374[s] | loss 2.26\n",
      "| epoch 1 |  iter 7581 / 9295 | time 375[s] | loss 2.28\n",
      "| epoch 1 |  iter 7601 / 9295 | time 376[s] | loss 2.24\n",
      "| epoch 1 |  iter 7621 / 9295 | time 377[s] | loss 2.25\n",
      "| epoch 1 |  iter 7641 / 9295 | time 378[s] | loss 2.27\n",
      "| epoch 1 |  iter 7661 / 9295 | time 379[s] | loss 2.25\n",
      "| epoch 1 |  iter 7681 / 9295 | time 380[s] | loss 2.27\n",
      "| epoch 1 |  iter 7701 / 9295 | time 381[s] | loss 2.25\n",
      "| epoch 1 |  iter 7721 / 9295 | time 382[s] | loss 2.22\n",
      "| epoch 1 |  iter 7741 / 9295 | time 383[s] | loss 2.27\n",
      "| epoch 1 |  iter 7761 / 9295 | time 384[s] | loss 2.24\n",
      "| epoch 1 |  iter 7781 / 9295 | time 385[s] | loss 2.27\n",
      "| epoch 1 |  iter 7801 / 9295 | time 386[s] | loss 2.26\n",
      "| epoch 1 |  iter 7821 / 9295 | time 387[s] | loss 2.24\n",
      "| epoch 1 |  iter 7841 / 9295 | time 388[s] | loss 2.25\n",
      "| epoch 1 |  iter 7861 / 9295 | time 389[s] | loss 2.24\n",
      "| epoch 1 |  iter 7881 / 9295 | time 390[s] | loss 2.25\n",
      "| epoch 1 |  iter 7901 / 9295 | time 391[s] | loss 2.24\n",
      "| epoch 1 |  iter 7921 / 9295 | time 392[s] | loss 2.28\n",
      "| epoch 1 |  iter 7941 / 9295 | time 393[s] | loss 2.26\n",
      "| epoch 1 |  iter 7961 / 9295 | time 394[s] | loss 2.24\n",
      "| epoch 1 |  iter 7981 / 9295 | time 395[s] | loss 2.26\n",
      "| epoch 1 |  iter 8001 / 9295 | time 396[s] | loss 2.26\n",
      "| epoch 1 |  iter 8021 / 9295 | time 397[s] | loss 2.25\n",
      "| epoch 1 |  iter 8041 / 9295 | time 399[s] | loss 2.26\n",
      "| epoch 1 |  iter 8061 / 9295 | time 400[s] | loss 2.28\n",
      "| epoch 1 |  iter 8081 / 9295 | time 401[s] | loss 2.22\n",
      "| epoch 1 |  iter 8101 / 9295 | time 402[s] | loss 2.26\n",
      "| epoch 1 |  iter 8121 / 9295 | time 403[s] | loss 2.25\n",
      "| epoch 1 |  iter 8141 / 9295 | time 404[s] | loss 2.26\n",
      "| epoch 1 |  iter 8161 / 9295 | time 405[s] | loss 2.25\n",
      "| epoch 1 |  iter 8181 / 9295 | time 406[s] | loss 2.27\n",
      "| epoch 1 |  iter 8201 / 9295 | time 407[s] | loss 2.24\n",
      "| epoch 1 |  iter 8221 / 9295 | time 408[s] | loss 2.27\n",
      "| epoch 1 |  iter 8241 / 9295 | time 409[s] | loss 2.24\n",
      "| epoch 1 |  iter 8261 / 9295 | time 410[s] | loss 2.26\n",
      "| epoch 1 |  iter 8281 / 9295 | time 411[s] | loss 2.24\n",
      "| epoch 1 |  iter 8301 / 9295 | time 412[s] | loss 2.29\n",
      "| epoch 1 |  iter 8321 / 9295 | time 413[s] | loss 2.29\n",
      "| epoch 1 |  iter 8341 / 9295 | time 414[s] | loss 2.25\n",
      "| epoch 1 |  iter 8361 / 9295 | time 415[s] | loss 2.23\n",
      "| epoch 1 |  iter 8381 / 9295 | time 416[s] | loss 2.21\n",
      "| epoch 1 |  iter 8401 / 9295 | time 417[s] | loss 2.24\n",
      "| epoch 1 |  iter 8421 / 9295 | time 418[s] | loss 2.25\n",
      "| epoch 1 |  iter 8441 / 9295 | time 419[s] | loss 2.25\n",
      "| epoch 1 |  iter 8461 / 9295 | time 420[s] | loss 2.24\n",
      "| epoch 1 |  iter 8481 / 9295 | time 421[s] | loss 2.27\n",
      "| epoch 1 |  iter 8501 / 9295 | time 422[s] | loss 2.24\n",
      "| epoch 1 |  iter 8521 / 9295 | time 423[s] | loss 2.25\n",
      "| epoch 1 |  iter 8541 / 9295 | time 424[s] | loss 2.24\n",
      "| epoch 1 |  iter 8561 / 9295 | time 425[s] | loss 2.23\n",
      "| epoch 1 |  iter 8581 / 9295 | time 426[s] | loss 2.22\n",
      "| epoch 1 |  iter 8601 / 9295 | time 427[s] | loss 2.22\n",
      "| epoch 1 |  iter 8621 / 9295 | time 427[s] | loss 2.24\n",
      "| epoch 1 |  iter 8641 / 9295 | time 428[s] | loss 2.24\n",
      "| epoch 1 |  iter 8661 / 9295 | time 429[s] | loss 2.25\n",
      "| epoch 1 |  iter 8681 / 9295 | time 430[s] | loss 2.22\n",
      "| epoch 1 |  iter 8701 / 9295 | time 431[s] | loss 2.21\n",
      "| epoch 1 |  iter 8721 / 9295 | time 432[s] | loss 2.29\n",
      "| epoch 1 |  iter 8741 / 9295 | time 433[s] | loss 2.27\n",
      "| epoch 1 |  iter 8761 / 9295 | time 434[s] | loss 2.23\n",
      "| epoch 1 |  iter 8781 / 9295 | time 435[s] | loss 2.23\n",
      "| epoch 1 |  iter 8801 / 9295 | time 436[s] | loss 2.25\n",
      "| epoch 1 |  iter 8821 / 9295 | time 437[s] | loss 2.19\n",
      "| epoch 1 |  iter 8841 / 9295 | time 438[s] | loss 2.22\n",
      "| epoch 1 |  iter 8861 / 9295 | time 439[s] | loss 2.22\n",
      "| epoch 1 |  iter 8881 / 9295 | time 440[s] | loss 2.23\n",
      "| epoch 1 |  iter 8901 / 9295 | time 440[s] | loss 2.21\n",
      "| epoch 1 |  iter 8921 / 9295 | time 441[s] | loss 2.24\n",
      "| epoch 1 |  iter 8941 / 9295 | time 442[s] | loss 2.23\n",
      "| epoch 1 |  iter 8961 / 9295 | time 443[s] | loss 2.22\n",
      "| epoch 1 |  iter 8981 / 9295 | time 444[s] | loss 2.23\n",
      "| epoch 1 |  iter 9001 / 9295 | time 445[s] | loss 2.23\n",
      "| epoch 1 |  iter 9021 / 9295 | time 446[s] | loss 2.22\n",
      "| epoch 1 |  iter 9041 / 9295 | time 447[s] | loss 2.21\n",
      "| epoch 1 |  iter 9061 / 9295 | time 448[s] | loss 2.24\n",
      "| epoch 1 |  iter 9081 / 9295 | time 449[s] | loss 2.21\n",
      "| epoch 1 |  iter 9101 / 9295 | time 450[s] | loss 2.20\n",
      "| epoch 1 |  iter 9121 / 9295 | time 451[s] | loss 2.24\n",
      "| epoch 1 |  iter 9141 / 9295 | time 452[s] | loss 2.20\n",
      "| epoch 1 |  iter 9161 / 9295 | time 453[s] | loss 2.23\n",
      "| epoch 1 |  iter 9181 / 9295 | time 453[s] | loss 2.20\n",
      "| epoch 1 |  iter 9201 / 9295 | time 454[s] | loss 2.22\n",
      "| epoch 1 |  iter 9221 / 9295 | time 455[s] | loss 2.23\n",
      "| epoch 1 |  iter 9241 / 9295 | time 456[s] | loss 2.20\n",
      "| epoch 1 |  iter 9261 / 9295 | time 457[s] | loss 2.22\n",
      "| epoch 1 |  iter 9281 / 9295 | time 458[s] | loss 2.21\n",
      "| epoch 2 |  iter 1 / 9295 | time 459[s] | loss 2.21\n",
      "| epoch 2 |  iter 21 / 9295 | time 460[s] | loss 2.19\n",
      "| epoch 2 |  iter 41 / 9295 | time 461[s] | loss 2.17\n",
      "| epoch 2 |  iter 61 / 9295 | time 462[s] | loss 2.19\n",
      "| epoch 2 |  iter 81 / 9295 | time 463[s] | loss 2.17\n",
      "| epoch 2 |  iter 101 / 9295 | time 464[s] | loss 2.19\n",
      "| epoch 2 |  iter 121 / 9295 | time 465[s] | loss 2.17\n",
      "| epoch 2 |  iter 141 / 9295 | time 466[s] | loss 2.15\n",
      "| epoch 2 |  iter 161 / 9295 | time 467[s] | loss 2.15\n",
      "| epoch 2 |  iter 181 / 9295 | time 468[s] | loss 2.18\n",
      "| epoch 2 |  iter 201 / 9295 | time 469[s] | loss 2.17\n",
      "| epoch 2 |  iter 221 / 9295 | time 470[s] | loss 2.19\n",
      "| epoch 2 |  iter 241 / 9295 | time 471[s] | loss 2.17\n",
      "| epoch 2 |  iter 261 / 9295 | time 472[s] | loss 2.16\n",
      "| epoch 2 |  iter 281 / 9295 | time 473[s] | loss 2.16\n",
      "| epoch 2 |  iter 301 / 9295 | time 474[s] | loss 2.17\n",
      "| epoch 2 |  iter 321 / 9295 | time 475[s] | loss 2.14\n",
      "| epoch 2 |  iter 341 / 9295 | time 477[s] | loss 2.18\n",
      "| epoch 2 |  iter 361 / 9295 | time 478[s] | loss 2.18\n",
      "| epoch 2 |  iter 381 / 9295 | time 479[s] | loss 2.18\n",
      "| epoch 2 |  iter 401 / 9295 | time 480[s] | loss 2.17\n",
      "| epoch 2 |  iter 421 / 9295 | time 481[s] | loss 2.19\n",
      "| epoch 2 |  iter 441 / 9295 | time 482[s] | loss 2.18\n",
      "| epoch 2 |  iter 461 / 9295 | time 483[s] | loss 2.21\n",
      "| epoch 2 |  iter 481 / 9295 | time 484[s] | loss 2.13\n",
      "| epoch 2 |  iter 501 / 9295 | time 485[s] | loss 2.12\n",
      "| epoch 2 |  iter 521 / 9295 | time 486[s] | loss 2.15\n",
      "| epoch 2 |  iter 541 / 9295 | time 487[s] | loss 2.20\n",
      "| epoch 2 |  iter 561 / 9295 | time 488[s] | loss 2.15\n",
      "| epoch 2 |  iter 581 / 9295 | time 489[s] | loss 2.18\n",
      "| epoch 2 |  iter 601 / 9295 | time 490[s] | loss 2.19\n",
      "| epoch 2 |  iter 621 / 9295 | time 491[s] | loss 2.18\n",
      "| epoch 2 |  iter 641 / 9295 | time 492[s] | loss 2.17\n",
      "| epoch 2 |  iter 661 / 9295 | time 493[s] | loss 2.18\n",
      "| epoch 2 |  iter 681 / 9295 | time 494[s] | loss 2.16\n",
      "| epoch 2 |  iter 701 / 9295 | time 495[s] | loss 2.17\n",
      "| epoch 2 |  iter 721 / 9295 | time 496[s] | loss 2.16\n",
      "| epoch 2 |  iter 741 / 9295 | time 497[s] | loss 2.17\n",
      "| epoch 2 |  iter 761 / 9295 | time 498[s] | loss 2.16\n",
      "| epoch 2 |  iter 781 / 9295 | time 499[s] | loss 2.18\n",
      "| epoch 2 |  iter 801 / 9295 | time 500[s] | loss 2.17\n",
      "| epoch 2 |  iter 821 / 9295 | time 501[s] | loss 2.19\n",
      "| epoch 2 |  iter 841 / 9295 | time 502[s] | loss 2.13\n",
      "| epoch 2 |  iter 861 / 9295 | time 504[s] | loss 2.17\n",
      "| epoch 2 |  iter 881 / 9295 | time 505[s] | loss 2.14\n",
      "| epoch 2 |  iter 901 / 9295 | time 506[s] | loss 2.14\n",
      "| epoch 2 |  iter 921 / 9295 | time 507[s] | loss 2.18\n",
      "| epoch 2 |  iter 941 / 9295 | time 508[s] | loss 2.18\n",
      "| epoch 2 |  iter 961 / 9295 | time 509[s] | loss 2.16\n",
      "| epoch 2 |  iter 981 / 9295 | time 510[s] | loss 2.18\n",
      "| epoch 2 |  iter 1001 / 9295 | time 511[s] | loss 2.13\n",
      "| epoch 2 |  iter 1021 / 9295 | time 512[s] | loss 2.16\n",
      "| epoch 2 |  iter 1041 / 9295 | time 513[s] | loss 2.17\n",
      "| epoch 2 |  iter 1061 / 9295 | time 514[s] | loss 2.18\n",
      "| epoch 2 |  iter 1081 / 9295 | time 515[s] | loss 2.16\n",
      "| epoch 2 |  iter 1101 / 9295 | time 516[s] | loss 2.17\n",
      "| epoch 2 |  iter 1121 / 9295 | time 517[s] | loss 2.19\n",
      "| epoch 2 |  iter 1141 / 9295 | time 518[s] | loss 2.18\n",
      "| epoch 2 |  iter 1161 / 9295 | time 519[s] | loss 2.13\n",
      "| epoch 2 |  iter 1181 / 9295 | time 520[s] | loss 2.15\n",
      "| epoch 2 |  iter 1201 / 9295 | time 521[s] | loss 2.14\n",
      "| epoch 2 |  iter 1221 / 9295 | time 522[s] | loss 2.15\n",
      "| epoch 2 |  iter 1241 / 9295 | time 523[s] | loss 2.18\n",
      "| epoch 2 |  iter 1261 / 9295 | time 524[s] | loss 2.13\n",
      "| epoch 2 |  iter 1281 / 9295 | time 525[s] | loss 2.16\n",
      "| epoch 2 |  iter 1301 / 9295 | time 526[s] | loss 2.15\n",
      "| epoch 2 |  iter 1321 / 9295 | time 527[s] | loss 2.18\n",
      "| epoch 2 |  iter 1341 / 9295 | time 528[s] | loss 2.17\n",
      "| epoch 2 |  iter 1361 / 9295 | time 529[s] | loss 2.17\n",
      "| epoch 2 |  iter 1381 / 9295 | time 530[s] | loss 2.13\n",
      "| epoch 2 |  iter 1401 / 9295 | time 531[s] | loss 2.15\n",
      "| epoch 2 |  iter 1421 / 9295 | time 532[s] | loss 2.17\n",
      "| epoch 2 |  iter 1441 / 9295 | time 533[s] | loss 2.12\n",
      "| epoch 2 |  iter 1461 / 9295 | time 535[s] | loss 2.13\n",
      "| epoch 2 |  iter 1481 / 9295 | time 536[s] | loss 2.14\n",
      "| epoch 2 |  iter 1501 / 9295 | time 537[s] | loss 2.13\n",
      "| epoch 2 |  iter 1521 / 9295 | time 538[s] | loss 2.15\n",
      "| epoch 2 |  iter 1541 / 9295 | time 539[s] | loss 2.10\n",
      "| epoch 2 |  iter 1561 / 9295 | time 540[s] | loss 2.17\n",
      "| epoch 2 |  iter 1581 / 9295 | time 541[s] | loss 2.17\n",
      "| epoch 2 |  iter 1601 / 9295 | time 542[s] | loss 2.17\n",
      "| epoch 2 |  iter 1621 / 9295 | time 543[s] | loss 2.16\n",
      "| epoch 2 |  iter 1641 / 9295 | time 544[s] | loss 2.13\n",
      "| epoch 2 |  iter 1661 / 9295 | time 545[s] | loss 2.14\n",
      "| epoch 2 |  iter 1681 / 9295 | time 546[s] | loss 2.13\n",
      "| epoch 2 |  iter 1701 / 9295 | time 547[s] | loss 2.14\n",
      "| epoch 2 |  iter 1721 / 9295 | time 548[s] | loss 2.13\n",
      "| epoch 2 |  iter 1741 / 9295 | time 549[s] | loss 2.16\n",
      "| epoch 2 |  iter 1761 / 9295 | time 550[s] | loss 2.16\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mKeyboardInterrupt\u001B[0m                         Traceback (most recent call last)",
      "Cell \u001B[1;32mIn[24], line 2\u001B[0m\n\u001B[0;32m      1\u001B[0m \u001B[38;5;66;03m# 开始学习\u001B[39;00m\n\u001B[1;32m----> 2\u001B[0m \u001B[43mtrainer\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mfit\u001B[49m\u001B[43m(\u001B[49m\u001B[43mcontexts\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mtarget\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmax_epoch\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mbatch_size\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m      3\u001B[0m trainer\u001B[38;5;241m.\u001B[39mplot()\n",
      "File \u001B[1;32mD:\\python2\\code7\\NLP\\01_深度学习进阶：自然语言处理\\4_word2vec的高速化\\..\\common\\trainer.py:40\u001B[0m, in \u001B[0;36mTrainer.fit\u001B[1;34m(self, x, t, max_epoch, batch_size, max_grad, eval_interval)\u001B[0m\n\u001B[0;32m     38\u001B[0m \u001B[38;5;66;03m# 计算梯度，更新参数\u001B[39;00m\n\u001B[0;32m     39\u001B[0m loss \u001B[38;5;241m=\u001B[39m model\u001B[38;5;241m.\u001B[39mforward(batch_x, batch_t)\n\u001B[1;32m---> 40\u001B[0m \u001B[43mmodel\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mbackward\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m     41\u001B[0m params, grads \u001B[38;5;241m=\u001B[39m remove_duplicate(model\u001B[38;5;241m.\u001B[39mparams, model\u001B[38;5;241m.\u001B[39mgrads)  \u001B[38;5;66;03m# 将共享的权重整合为1个\u001B[39;00m\n\u001B[0;32m     42\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m max_grad \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n",
      "Cell \u001B[1;32mIn[3], line 41\u001B[0m, in \u001B[0;36mCBOW.backward\u001B[1;34m(self, dout)\u001B[0m\n\u001B[0;32m     39\u001B[0m dout \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m=\u001B[39m \u001B[38;5;241m1\u001B[39m \u001B[38;5;241m/\u001B[39m \u001B[38;5;28mlen\u001B[39m(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39min_layers)\n\u001B[0;32m     40\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m layer \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39min_layers:\n\u001B[1;32m---> 41\u001B[0m     \u001B[43mlayer\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mbackward\u001B[49m\u001B[43m(\u001B[49m\u001B[43mdout\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m     42\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m\n",
      "File \u001B[1;32mD:\\python2\\code7\\NLP\\01_深度学习进阶：自然语言处理\\4_word2vec的高速化\\..\\common\\layers.py:169\u001B[0m, in \u001B[0;36mEmbedding.backward\u001B[1;34m(self, dout)\u001B[0m\n\u001B[0;32m    167\u001B[0m     np\u001B[38;5;241m.\u001B[39mscatter_add(dW, \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39midx, dout)\n\u001B[0;32m    168\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m--> 169\u001B[0m     \u001B[43mnp\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43madd\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mat\u001B[49m\u001B[43m(\u001B[49m\u001B[43mdW\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43midx\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mdout\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    170\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m\n",
      "\u001B[1;31mKeyboardInterrupt\u001B[0m: "
     ]
    }
   ],
   "source": [
    "# 开始学习\n",
    "trainer.fit(contexts, target, max_epoch, batch_size)\n",
    "trainer.plot()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-06T16:53:02.611606100Z",
     "start_time": "2023-05-06T16:43:51.538601100Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "outputs": [],
   "source": [
    "# 保存必要数据，以便后续使用\n",
    "word_vecs = model.word_vecs\n",
    "if config.GPU:\n",
    "    word_vecs = to_cpu(word_vecs)\n",
    "params = {}\n",
    "params['word_vecs'] = word_vecs.astype(np.float16)\n",
    "params['word_to_id'] = word_to_id\n",
    "params['id_to_word'] = id_to_word\n",
    "pkl_file = 'cbow_params.pkl'\n",
    "with open(pkl_file, 'wb') as f:\n",
    "    pickle.dump(params, f, -1)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-06T16:53:34.624622100Z",
     "start_time": "2023-05-06T16:53:34.612624300Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "outputs": [],
   "source": [
    "from common.util import most_similar"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-06T16:54:13.866726700Z",
     "start_time": "2023-05-06T16:54:13.844731100Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[query] you\n",
      " why: 0.974609375\n",
      " we: 0.97314453125\n",
      " i: 0.97021484375\n",
      " please: 0.95703125\n",
      " something: 0.94921875\n",
      "\n",
      "[query] year\n",
      " month: 0.9365234375\n",
      " week: 0.8974609375\n",
      " earlier: 0.83154296875\n",
      " period: 0.82421875\n",
      " forecast: 0.81884765625\n",
      "\n",
      "[query] car\n",
      " contest: 0.9453125\n",
      " injection: 0.94287109375\n",
      " sedan: 0.93798828125\n",
      " disk: 0.92724609375\n",
      " army: 0.92333984375\n",
      "\n",
      "[query] toyota\n",
      " ind.: 0.9375\n",
      " penney: 0.93115234375\n",
      " northrop: 0.92919921875\n",
      " proposes: 0.92724609375\n",
      " kodak: 0.923828125\n"
     ]
    }
   ],
   "source": [
    "pkl_file = 'cbow_params.pkl'\n",
    "with open(pkl_file, 'rb') as f:\n",
    "    params = pickle.load(f)\n",
    "    word_vecs = params['word_vecs']\n",
    "    word_to_id = params['word_to_id']\n",
    "    id_to_word = params['id_to_word']\n",
    "\n",
    "querys = ['you', 'year', 'car', 'toyota']\n",
    "for query in querys:\n",
    "    most_similar(query, word_to_id, id_to_word, word_vecs, top=5)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-06T16:54:15.257318300Z",
     "start_time": "2023-05-06T16:54:14.360324700Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "outputs": [],
   "source": [
    "from common.util import analogy"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-06T16:54:25.304453700Z",
     "start_time": "2023-05-06T16:54:25.279431400Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[analogy] king:man = queen:?\n",
      " share: 5.0078125\n",
      " thing: 4.81640625\n",
      " spokesman: 4.74609375\n",
      " way: 4.58203125\n",
      " lot: 4.54296875\n"
     ]
    }
   ],
   "source": [
    "analogy('king', 'man', 'queen', word_to_id, id_to_word, word_vecs)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-06T16:54:26.495911Z",
     "start_time": "2023-05-06T16:54:26.448965200Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[analogy] take:took = go:?\n",
      " composite: 5.7265625\n",
      " york: 4.66796875\n",
      " exchange: 4.5234375\n",
      " quarter: 4.296875\n",
      " ended: 4.08984375\n"
     ]
    }
   ],
   "source": [
    "analogy('take', 'took', 'go', word_to_id, id_to_word, word_vecs)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-06T16:54:27.281391400Z",
     "start_time": "2023-05-06T16:54:27.263395600Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[analogy] car:cars = child:?\n",
      " yield: 4.88671875\n",
      " addition: 4.2421875\n",
      " come: 4.171875\n",
      " revenue: 4.08984375\n",
      " try: 3.91015625\n"
     ]
    }
   ],
   "source": [
    "analogy('car', 'cars', 'child', word_to_id, id_to_word, word_vecs)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-06T16:54:27.792970100Z",
     "start_time": "2023-05-06T16:54:27.741974500Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[analogy] good:better = bad:?\n",
      " do: 4.0\n",
      " know: 3.912109375\n",
      " than: 3.8515625\n",
      " think: 3.833984375\n",
      " n't: 3.75\n"
     ]
    }
   ],
   "source": [
    "analogy('good', 'better', 'bad', word_to_id, id_to_word, word_vecs)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-06T16:54:28.324427900Z",
     "start_time": "2023-05-06T16:54:28.276427800Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "模型训练的时间太少，结果正确率低\n",
    "\"\"\""
   ],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
